package navigation.results.cluster;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileFilter;
import java.io.FileNotFoundException;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;

import org.apache.commons.lang3.StringEscapeUtils;
import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.filters.HasAttributeFilter;
import org.htmlparser.lexer.Lexer;
import org.htmlparser.lexer.Page;
import org.htmlparser.nodes.TagNode;
import org.htmlparser.nodes.TextNode;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

import util.nodecloner.ChildFinder;
import wish.stages.ParsingStage;

public class ClusterMaker {

	public class ClusterMakerRunnable implements Runnable{

		private static final String HREF_ATTRIBUTE = "href";
		private File input;
		private File output;
		

		public ClusterMakerRunnable(File file, File outp) {
			
			this.input = file;
			this.output = outp;
			
		}

		@Override
		public void run() {
						
			try {
				
				String htmlContent = ParsingStage.readFile(input);
				
				Parser parser = new Parser(new Lexer(new Page(htmlContent, "UTF-8")));
				
				NodeList n = parser.parse(new HasAttributeFilter(HREF_ATTRIBUTE));
				
				Collection<NodeList> clusters = clusterByName(n);				
				
				BufferedWriter bw = new BufferedWriter(new FileWriter(output));
				
				boolean theClusterFound = false;
				
				int numberOfTheCluster = 1;
				
				for (NodeList nodeList : clusters) {
					
					bw.write("<div");
					
					if (isTheCluster(nodeList)){
						bw.write(" id=\""+ numberOfTheCluster+ "\"><h1> THE CLUSTER: </h1>\n");
						theClusterFound = true;
						numberOfTheCluster++;
					}else{
						bw.write(">\n<h2> Cluster: </h2>\n");
					}
										
					for (int i = 0; i < nodeList.size(); i++) {
						
						Node node = nodeList.elementAt(i);
						
						bw.write("<br>");
						
						bw.write(node.toHtml());
						
					}
					bw.write("</div>");
				
				}
				
				if (!theClusterFound){
					
					System.err.println(output.getAbsolutePath());
					
				}
				
				bw.close();
			} catch (ParserException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			} catch (IOException e) {
				// TODO Auto-generated catch block
				e.printStackTrace();
			}
			
		}

		private boolean isTheCluster(NodeList nodeList) {
			
			//based in numbers
			
			int size = nodeList.size();
			
			if (size <= 2){
				return false;
			}
			
			int allnumber = 0;
			
			ArrayList<String> texts = new ArrayList<String>();
			
			for (int i = 0; i < size; i++) {
				
				texts.add(ParsingStage.recoverText(nodeList.elementAt(i)));
				
			}
			
			
			for (String text : texts) {
				
				allnumber += AllNumbers(text.toCharArray());
				
			}
			
			if (allnumber==0)
				return false;
			
			boolean ret =  allnumber+2>=size;
			
			return ret;
						
		}

		private int AllNumbers(char[] cs) {
			
			if (cs.length == 0)
				return 0;
			
			for (int i = 0; i < cs.length; i++) {
				if (!Character.isDigit(cs[i]))
					return 0;
			}
			return 1;
		}

		private Collection<NodeList> clusterByName(NodeList n) {
			
			Map<String,NodeList> map = new HashMap<String, NodeList>();
			
			for (int i = 0; i < n.size(); i++) {
				
				TagNode t = (TagNode)n.elementAt(i);
				
				String hyperlink = t.getAttribute(HREF_ATTRIBUTE);
				
				String refined = generateKey(hyperlink);
				
				NodeList nl = map.get(refined);
				
				if (nl == null){
					nl = new NodeList();
					map.put(refined, nl);
				}
				
				//check if t is new.
				
				boolean exists = false;
				
				for (int j = 0; j < nl.size(); j++) {
					
					TagNode stored = (TagNode)n.elementAt(j);
					
					String ref = stored.getAttribute(HREF_ATTRIBUTE);
					
					if (ref.equals(hyperlink)){
						exists = true;
						break;
					}
						
					
				}
				
				if (!exists)
					nl.add(t);
				
			}
			
			return map.values();
			
		}

		private String generateKey(String hyperlink) {
			return hyperlink.replaceAll("[0-9]", "");
		}
		
	}

	private static Set<String> toProcess;
	
	/**
	 * @param args
	 * @throws IOException 
	 */
	public static void main(String[] args) throws IOException {
		
		toProcess = loadToProcess("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/NavigationReport/bad");
		
		String navFolder = "/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/Navigation/";
		
		File outp = new File("/proj/dbNoBackup/pjbarrio/Experiments/Wrappers/NavigationClusterComplexText/");
		
		File fold = new File(navFolder);
		
		File[] navfolders = fold.listFiles(new FileFilter() {
			
			@Override
			public boolean accept(File pathname) {
				return toProcess.contains(pathname.getName());
			}
		});
		
		//Process All
		
//		navfolders = fold.listFiles();
		
		for (int i = 0; i < navfolders.length; i++) {
			
			System.out.println("DATABASE: " + i);
			
			File folder = navfolders[i];
			
			String name = folder.getName();
			
			File[] files = folder.listFiles();
			
			File dir = new File(outp,name);
			
			if (dir.exists())
				continue;
			
			dir.mkdir();
			
			System.setErr(new PrintStream(new File(dir, "notFound.txt")));
			
			for (int j = 0; j < files.length; j++) {
				
				System.gc();
				
				System.out.println("FILE: " + j);
				
				new ClusterMaker().execute(files[j],new File(dir,files[j].getName()));
				
			}
			
		}

	}

	private static Set<String> loadToProcess(String fileName) throws IOException {
		
		Set<String> set = new HashSet<String>();
		
		BufferedReader br = new BufferedReader(new FileReader(fileName));
		
		String line;
		
		while((line=br.readLine())!=null){
			
			set.add(line);
			
		}
		
		br.close();
		
		return set;
		
	}

	private void execute(File file, File outp) {
		
		Thread t = new Thread(new ClusterMakerRunnable(file,outp));
		
		t.start();
		
		try {
			t.join();
		} catch (InterruptedException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		
	}

}
